package wish.stages;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.List;
import java.util.Set;
import java.util.Vector;


import org.htmlparser.Attribute;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.nodes.RemarkNode;
import org.htmlparser.nodes.TagNode;
import org.htmlparser.nodes.TextNode;
import org.htmlparser.tags.Div;
import org.htmlparser.tags.Html;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.htmlparser.util.SimpleNodeIterator;
import org.htmlparser.visitors.NodeVisitor;

import wish.model.Record;
import wish.model.TaggerExtractor;
import wish.stages.htmlFilter.AttributeContainKeywordsFilter;


public class DataExtractionStage {

	private static final int MIN_NUMBER_OF_OCCURENCES = 2;
	
	private List<Record> extractedRecords = new ArrayList<Record>();

	public List<Record> findRecords(Parser parser) throws ParserException{
		
		TagNode main = new Html();
		
//		NodeList n = parser.parse(new AttributeContainKeywordsFilter(keyWords));
			
		NodeIterator iterator = parser.elements();
		
		NodeList n = new NodeList();
		
		while (iterator.hasMoreNodes()){
			
			n.add(iterator.nextNode());
			
		}
		
		List<Record> ret = new ArrayList<Record>();

		int level = 0;
		
		List<Node> nodesLeft = processLevel(main,n,level);
		
		ret.addAll(extractedRecords);
		
		while(nodesLeft.size()>0){
			
			level++;
			
			List<Node> nodestoProcess = new ArrayList<Node>();
			
			for (Node node : nodesLeft) {
				
				n = node.getChildren();
				
				if (n != null){
				
					nodestoProcess.addAll(processLevel((TagNode)node,n,level));
					
					ret.addAll(extractedRecords);

				}
			
			}
			
			nodesLeft = nodestoProcess;

		}
				
		return ret;
		
	}

	private List<Node> processLevel(TagNode father, NodeList n,int level) {
		
		extractedRecords.clear();
		
		List<Node> nodesLeft = new ArrayList<Node>();
		
		Hashtable<String,NodeList> groupsbyTag = groupbyTag(n);
		
		for(Enumeration<String> e = groupsbyTag.keys(); e.hasMoreElements();){
			
			String tag = e.nextElement();
			
			NodeList groupbyTag = groupsbyTag.get(tag);
			
			if (groupbyTag.size()>=MIN_NUMBER_OF_OCCURENCES){
				extractedRecords.add(generateNewRecord(father,groupbyTag,level,tag));
			}

			SimpleNodeIterator iterator = groupbyTag.elements();
			
			while (iterator.hasMoreNodes()){
								
				Node node = iterator.nextNode();
				
				nodesLeft.add(node);
				
			}
			
		}
		
		return nodesLeft;
		
	}

	private Record generateNewRecord(TagNode father, NodeList groupBytag, int level, String tag) {
		
		return new Record(father,level,tag,groupBytag,groupBytag.size());
		
	}

	private Hashtable<String, NodeList> groupbyTag(NodeList n) {
		
		Hashtable<String, NodeList> ret = new Hashtable<String, NodeList>();
		
		SimpleNodeIterator iterator = n.elements();
		
		while (iterator.hasMoreNodes()){
			
			Node node = iterator.nextNode();
			
			String tag = TaggerExtractor.extractTag(node);
			
			NodeList list = ret.get(tag);
			
			if (list == null){
				list = new NodeList();
			}
			
			list.add(node);
			
			ret.put(tag, list);
		}
		
		return ret;
		
	}

	public static void main(String[] args) throws IOException, ParserException {
		
		String html1 = "/home/pjbarrio/Documents/RelationExtractionStuff/DatabasesInterfaces/ResultsPage/1.html";
		
		String html1Content = ParsingStage.readFile(new File(html1));
		
//		String html2 = "/home/pjbarrio/Documents/RelationExtractionStuff/DatabasesInterfaces/ResultsPage/2.html"
		
		List<String> htmlContent = new ArrayList<String>();
		
		htmlContent.add(html1Content);
		
		List<Parser> list = new ParsingStage().ParseWebSite(htmlContent);
		
		for (Parser parser : list) {
			
			List<Record> records = new DataExtractionStage().findRecords(parser);
			
			for (Record record : records) {
				
				System.out.println(record);
				
			}
			
		}
		
	}
	
}
